import requests
import json
from dbutil.pgsql import PgsqlPipeline
from datetime import date

def get_provinces():
	url = "https://www.denza.com/portal/dealer/prosales.html"
	r = requests.get(url)
	provinces = json.loads(r.text)
	return provinces

def get_cities(province_id):
	url = f"https://www.denza.com/portal/dealer/city.html?fid={province_id}"
	r = requests.get(url)
	cities = json.loads(r.text)
	return cities

def get_dealers(province_id, city_id):
	url = f"https://www.denza.com/portal/dealer/indexsales.html?sid=1&fid={province_id}&cid={city_id}"
	r = requests.get(url)
	dealers = json.loads(r.text)
	return dealers

def get_item(dealer, province_name, city_name):
	item = {
		"dealer_name" : dealer["name"],
		"brand_id" : None,
		"address" : dealer["addr"],
		"brand" : "腾势",
		"province" : province_name,
		"city" : city_name,
		"sale_call" : dealer["tel"],
		"customer_service_call" : None,
		"update_time" : date.today(),
		"longitude" : dealer["longitude"],
		"latitude" : dealer["latitude"],
		"dealer_type" : None,
		"manufacturer_id" : None,
		"manufacturer" : "腾势新能源",
		"state" : None,
		"opening_date" : None,
		"close_date" : None,
		"dealer_id_web" : None,
		"controlling_shareholder" : None,
		"other_shareholders" : None,
		"status" : None,
		"remarks" : None,
	}
	return item

def get_items():
	items = []

	provinces = get_provinces()
	for province in provinces:
		province_id = province["id"]
		province_name = province["area"]
		cities = get_cities(province_id)
		for city in cities:
			city_id = city["id"]
			city_name = city["area"]
			dealers = get_dealers(province_id, city_id)
			for dealer in dealers:
				item = get_item(dealer, province_name, city_name)
				items.append(item)
	return items


def main():
	print("爬虫开始...")
	items = get_items()
	print("数据存库")
	pg = PgsqlPipeline()
	for item in items:
		pg.process_item(item)
	pg.close()
	print("完成")

if __name__ == '__main__':
	main()